{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import Dense, Input, Flatten\n",
    "\n",
    "from keras.layers import GlobalMaxPool1D, Bidirectional, Convolution1D, Embedding, BatchNormalization, MaxPooling1D, \\\n",
    "    Dropout, LSTM\n",
    "\n",
    "\n",
    "from keras import backend as K\n",
    "\n",
    "from keras.engine.topology import Layer\n",
    "\n",
    "from keras import initializers, regularizers, constraints\n",
    "\n",
    "import numpy as np\n",
    "from keras.models import Model\n",
    "\n",
    "from keras.layers.merge import Concatenate\n",
    "\n",
    "\n",
    "\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "'''\n",
    "可以定义EarlyStopping来提前终止训练\n",
    "\n",
    "from keras.callbacks import EarlyStopping\n",
    "early_stopping = EarlyStopping(monitor='val_loss', patience=2)\n",
    "model.fit(X, y, validation_split=0.2, callbacks=[early_stopping])\n",
    "\n",
    "'''\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "r = np.load(\"./cache/data.npz\")  # 加载一次即可\n",
    "x_train = r['arr_0']\n",
    "y_train = r['arr_1']\n",
    "x_val = r['arr_2']\n",
    "y_val = r['arr_3']\n",
    "X_te = r['arr_4']\n",
    "embedding_matrix = r['arr_5']\n",
    "\n",
    "max_features = 20000\n",
    "MAX_SEQUENCE_LENGTH = 100\n",
    "EMBEDDING_DIM = 128\n",
    "\n",
    "num_lstm = 150\n",
    "rate_drop_lstm = 0.25\n",
    "rate_drop_dense = 0.25\n",
    "# 在最优方案5的基础上进行修改阈值4.7，分别将其改为4.6,4.7,4.8没有对其进行修改，之后还试验了\n",
    "# final_score[final_score>4.7] =5发现等于4.7的结果最优\n",
    "\n",
    "embedding_layer = Embedding(max_features,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False)\n",
    "\n",
    "lstm_layer = LSTM(num_lstm, dropout=rate_drop_lstm, recurrent_dropout=rate_drop_lstm, return_sequences=True)\n",
    "\n",
    "\n",
    "# 双向LSTM模型\n",
    "def get_lstm_model():\n",
    "    inp = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "    embedded_sequences = embedding_layer(inp)\n",
    "    x = Bidirectional(LSTM(250, dropout=rate_drop_lstm, recurrent_dropout=rate_drop_lstm, return_sequences=True))(\n",
    "        embedded_sequences)\n",
    "    x = GlobalMaxPool1D()(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "    x = Dense(100, activation=\"relu\")(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "    x = Dense(1, activation=\"linear\")(x)\n",
    "    model = Model(inputs=inp, outputs=x)\n",
    "    model.compile(loss='mse',\n",
    "                  optimizer='adam')\n",
    "    return model\n",
    "\n",
    "\n",
    "# attention model\n",
    "num_lstm = 300\n",
    "num_dense = 256\n",
    "rate_drop_lstm = 0.25\n",
    "rate_drop_dense = 0.25\n",
    "\n",
    "act = 'relu'\n",
    "\n",
    "\n",
    "class Attention(Layer):\n",
    "    def __init__(self, step_dim,\n",
    "                 W_regularizer=None, b_regularizer=None,\n",
    "                 W_constraint=None, b_constraint=None,\n",
    "                 bias=True, **kwargs):\n",
    "        \"\"\"\n",
    "        Keras Layer that implements an Attention mechanism for temporal data.\n",
    "        Supports Masking.\n",
    "        Follows the work of Raffel et al. [https://arxiv.org/abs/1512.08756]\n",
    "        # Input shape\n",
    "            3D tensor with shape: `(samples, steps, features)`.\n",
    "        # Output shape\n",
    "            2D tensor with shape: `(samples, features)`.\n",
    "        :param kwargs:\n",
    "        Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True.\n",
    "        The dimensions are inferred based on the output shape of the RNN.\n",
    "        Example:\n",
    "            model.add(LSTM(64, return_sequences=True))\n",
    "            model.add(Attention())\n",
    "        \"\"\"\n",
    "        self.supports_masking = True\n",
    "        self.init = initializers.get('glorot_uniform')\n",
    "\n",
    "        self.W_regularizer = regularizers.get(W_regularizer)\n",
    "        self.b_regularizer = regularizers.get(b_regularizer)\n",
    "\n",
    "        self.W_constraint = constraints.get(W_constraint)\n",
    "        self.b_constraint = constraints.get(b_constraint)\n",
    "\n",
    "        self.bias = bias\n",
    "        self.step_dim = step_dim\n",
    "        self.features_dim = 0\n",
    "        super(Attention, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        assert len(input_shape) == 3\n",
    "\n",
    "        self.W = self.add_weight((input_shape[-1],),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_W'.format(self.name),\n",
    "                                 regularizer=self.W_regularizer,\n",
    "                                 constraint=self.W_constraint)\n",
    "        self.features_dim = input_shape[-1]\n",
    "\n",
    "        if self.bias:\n",
    "            self.b = self.add_weight((input_shape[1],),\n",
    "                                     initializer='zero',\n",
    "                                     name='{}_b'.format(self.name),\n",
    "                                     regularizer=self.b_regularizer,\n",
    "                                     constraint=self.b_constraint)\n",
    "        else:\n",
    "            self.b = None\n",
    "\n",
    "        self.built = True\n",
    "\n",
    "    def compute_mask(self, input, input_mask=None):\n",
    "        # do not pass the mask to the next layers\n",
    "        return None\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        # eij = K.dot(x, self.W) TF backend doesn't support it\n",
    "\n",
    "        # features_dim = self.W.shape[0]\n",
    "        # step_dim = x._keras_shape[1]\n",
    "\n",
    "        features_dim = self.features_dim\n",
    "        step_dim = self.step_dim\n",
    "\n",
    "        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)), K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n",
    "\n",
    "        if self.bias:\n",
    "            eij += self.b\n",
    "\n",
    "        eij = K.tanh(eij)\n",
    "\n",
    "        a = K.exp(eij)\n",
    "\n",
    "        # apply mask after the exp. will be re-normalized next\n",
    "        if mask is not None:\n",
    "            # Cast the mask to floatX to avoid float64 upcasting in theano\n",
    "            a *= K.cast(mask, K.floatx())\n",
    "\n",
    "        # in some cases especially in the early stages of training the sum may be almost zero\n",
    "        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
    "\n",
    "        a = K.expand_dims(a)\n",
    "        weighted_input = x * a\n",
    "        # print weigthted_input.shape\n",
    "        return K.sum(weighted_input, axis=1)\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        # return input_shape[0], input_shape[-1]\n",
    "        return input_shape[0], self.features_dim\n",
    "\n",
    "\n",
    "def get_attention_model():\n",
    "    inp = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "    embedded_sequences = embedding_layer(inp)\n",
    "    x = lstm_layer(embedded_sequences)\n",
    "    x = Dropout(rate_drop_dense)(x)\n",
    "    merged = Attention(MAX_SEQUENCE_LENGTH)(x)\n",
    "    merged = Dense(num_dense, activation=act)(merged)\n",
    "    merged = Dropout(rate_drop_dense)(merged)\n",
    "    merged = BatchNormalization()(merged)\n",
    "    preds = Dense(1, activation='linear')(merged)\n",
    "    model = Model(inputs=inp, outputs=preds)\n",
    "    model.compile(loss='mse',\n",
    "                  optimizer='adam')\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "# CNN模型\n",
    "\n",
    "filter_sizes = (2, 3, 4, 5)\n",
    "num_filters = 10\n",
    "dropout_prob = (0.1, 0.1)\n",
    "\n",
    "\n",
    "def get_CNN_model():\n",
    "    inp = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "    x = embedding_layer(inp)\n",
    "    conv_blocks = []\n",
    "    for sz in filter_sizes:\n",
    "        conv = Convolution1D(filters=num_filters,\n",
    "                             kernel_size=sz,\n",
    "                             padding=\"valid\",\n",
    "                             activation=\"relu\",\n",
    "                             strides=1)(x)\n",
    "        conv = MaxPooling1D(pool_size=2)(conv)\n",
    "        conv = Flatten()(conv)\n",
    "        conv_blocks.append(conv)\n",
    "    x = Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dense(50, activation=\"relu\")(x)\n",
    "    x = Dense(1, activation=\"linear\")(x)\n",
    "    model = Model(inputs=inp, outputs=x)\n",
    "    model.compile(loss='mse',\n",
    "                  optimizer='adam')\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "def get_model(model_name):\n",
    "    if model_name == 'RNN':\n",
    "        return get_lstm_model()\n",
    "    elif model_name == 'Attention':\n",
    "        return get_attention_model()\n",
    "    elif model_name == 'CNN':\n",
    "        return get_CNN_model()\n",
    "\n",
    "\n",
    "def predict(model_name):\n",
    "    model = get_model(model_name)\n",
    "    batch_size = 128\n",
    "    epochs = 100\n",
    "\n",
    "    file_path = \"./cache/weights_base.best.hdf5\"\n",
    "    checkpoint = ModelCheckpoint(file_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\n",
    "\n",
    "    early = EarlyStopping(monitor=\"val_loss\", mode=\"min\", patience=5)\n",
    "\n",
    "    callbacks_list = [checkpoint, early]\n",
    "    model.fit(x_train, y_train, batch_size=batch_size, epochs=100, validation_data=(x_val, y_val),\n",
    "              callbacks=callbacks_list)\n",
    "\n",
    "    model.load_weights(file_path)\n",
    "\n",
    "    y_test = model.predict(X_te)\n",
    "\n",
    "    y_test[y_test > 5] = 5\n",
    "\n",
    "    test = pd.read_csv('./cache/predict-processed.csv')\n",
    "    sub = pd.DataFrame()\n",
    "    sub['id'] = pd.DataFrame(test[\"Id\"])\n",
    "    sub['Score'] = pd.DataFrame(y_test)\n",
    "    sub.to_csv('./cache/sub_{}.csv'.format(model_name), index=False, header=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model  = get_CNN_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 100)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 100, 128)     2560000     input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)               (None, 99, 10)       2570        embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)               (None, 98, 10)       3850        embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_3 (Conv1D)               (None, 97, 10)       5130        embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_4 (Conv1D)               (None, 96, 10)       6410        embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1D)  (None, 49, 10)       0           conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1D)  (None, 49, 10)       0           conv1d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_3 (MaxPooling1D)  (None, 48, 10)       0           conv1d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_4 (MaxPooling1D)  (None, 48, 10)       0           conv1d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 490)          0           max_pooling1d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_2 (Flatten)             (None, 490)          0           max_pooling1d_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_3 (Flatten)             (None, 480)          0           max_pooling1d_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_4 (Flatten)             (None, 480)          0           max_pooling1d_4[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 1940)         0           flatten_1[0][0]                  \n",
      "                                                                 flatten_2[0][0]                  \n",
      "                                                                 flatten_3[0][0]                  \n",
      "                                                                 flatten_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 1940)         7760        concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 50)           97050       batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 1)            51          dense_1[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 2,682,821\n",
      "Trainable params: 118,941\n",
      "Non-trainable params: 2,563,880\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.utils.vis_utils import plot_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plot_model(model,to_file=\"cnnmodel.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "attention_model = get_attention_model()\n",
    "plot_model(attention_model,to_file=\"attention_model.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rnn_model = get_lstm_model()\n",
    "plot_model(rnn_model,to_file=\"rnn_model.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
